# -*- coding: utf-8 -*-

BOT_NAME = 'job51'

SPIDER_MODULES = ['job51.spiders']
NEWSPIDER_MODULE = 'job51.spiders'

ROBOTSTXT_OBEY = False

USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:61.0) Gecko/20100101 Firefox/61.0"

import logging
logger = logging.getLogger()
logger.addHandler(logging.StreamHandler())
logger.addHandler(logging.FileHandler('C:/Users/hqs/Desktop/script/python/spider/bak/test.log', mode='w', encoding='utf-8'))
logger.setLevel(logging.CRITICAL)  # 输出所有大于INFO级别的log

ITEM_PIPELINES = {
   'job51.pipelines.job51Pipeline': 300,
}

PROXY_URL = 'http://localhost:5555/random'

DOWNLOADER_MIDDLEWARES = {
    'job51.middlewares.ProxyMiddleware': 555,
}